<?php
set_time_limit(0);
error_reporting(E_ALL & ~E_NOTICE);
require_once('./vendor/autoload.php');
require_once('./config.php');

use QL\QueryList;
use QL\Ext\AbsoluteUrl;
use QL\Ext\CurlMulti;

$baseUrl = 'https://www.yp.mo';

$start = [
    // 最大并发数，这个值可以运行中动态改变。
    'maxThread' => 10,
    // 触发curl错误或用户错误之前最大重试次数，超过次数$error指定的回调会被调用。
    'maxTry' => 3,
    // 全局CURLOPT_*
    'opt' => [
        CURLOPT_TIMEOUT => 10,
        CURLOPT_CONNECTTIMEOUT => 1,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_SSL_VERIFYPEER => false
    ],
    'cache' => [
        'enable' => true, 
        'compress' => false, 
        'dir' => __DIR__ . '/cache', 
        'expire' => 86400, 
        'verifyPost' => false
    ]
];

try {
    // 链接数据库
    $database = new Nette\Database\Connection(DB_DSN, DB_USER, DB_PASS);

    // 注册插件
    $ql = QueryList::use([
        AbsoluteUrl::class, // 转换URL相对路径到绝对路径
        CurlMulti::class    // Curl多线程采集
    ]);

    //获取分类链接
    $urls = $ql->get($baseUrl)->absoluteUrl($baseUrl)->find('.group li a')->attrs('href')->all();
    $ql->rules([
        'link' => ['.detail dl dt a','href']
    ])
    ->curlMulti($urls)
    ->success(function (QueryList $ql,CurlMulti $curl,$r) use ($baseUrl, $start, $database) {
        $curUrl= $r['info']['url'];
        echo "Current category url:{$curUrl} \r\n";
        if(strpos($curUrl, 'page') === false){
            $html = $ql->getHtml();
            preg_match('/totalPage: parseInt\("(\d+)"\)/',$html,$arr);
            $totalPage = $arr[1];
            if($totalPage){
                // 分页链接
                for ($i=2; $i <= $totalPage; $i++) { 
                    $curl->add($curUrl . '?page=' . $i);
                }
            }
        }
        $data = $ql->absoluteUrl($baseUrl)->query()->getData();

        if(count($data->all())){
            //这里采集详情入库
            $q = (new QueryList)->use(CurlMulti::class);
            $q->rules([
                'name' => ['a.no-under','text'],
                'category' => ['#categorieName', 'text'],
                'sub_category' => ['#subCategorieName', 'text'],
                'address' => ['.detail dd:eq(0)','text'],
                'address2' => ['.detail dd:eq(1)','text'],
                'tel' => ['[title^=電話]','text', '', function($tel){
                    return trim(explode('：',$tel)[1]);
                }],
                'fax' => ['[title^=傳真]','text', '', function($fax){
                    return trim(explode('：',$fax)[1]);
                }],
                'email' => ['[title^=電子郵件]','text', '', function($email){
                    return trim(explode('：',$email)[1]);
                }],
                'url' => ['[title^=網址] a','text'],
                'facebook' => ['[title^=Facebook] a', 'text'],
                'content' => ['#bSurvey','html', '', function($content){
                    return $content ? htmlspecialchars($content) : '';
                }],
                'logo' => ['.bussness-LOGO .t-c img','src'],
                'images' => ['.meta-carousel-thumb', 'html', '', function($item){
                    if($item){
                        $images = QueryList::html($item)->find('img')->attrs('src')->all();
                        if($images){
                            return implode(',',$images);
                        }
                    }
                    return '';
                }],
                'long' => ['body', 'html', '', function($content){
                    if($content){
                        preg_match('/businessLongitude = "(\S+)"/',$content, $arr);
                        return isset($arr[1]) ? trim($arr[1]) : '';
                    }
                }],
                'lat' => ['body', 'html', '', function($content){
                    if($content){
                        preg_match('/businessLatitude = "(\S+)"/',$content, $arr);
                        return isset($arr[1]) ? trim($arr[1]) : '';
                    }
                }]
            ])
            ->curlMulti(array_column($data->all(),'link'))
            ->success(function(QueryList $q,CurlMulti $c, $rs) use($database) {
                $curUrl= $rs['info']['url'];
                echo "Current details url:{$curUrl} \r\n";
                $data = $q->query()->getData();
                $q->destruct();
                $data = current($data->all());
                $result = $database->query('SELECT id FROM huangye WHERE `name` = ?', $data['name'])->fetch();
                if(!isset($result->id)){
                    $database->query('INSERT INTO huangye', $data);
                }else{
                    if(IS_UPDATE){
                        $database->query('UPDATE huangye SET ? WHERE `name`= ?', $data, $data['name']);
                    }
                }
            })
            ->start($start);
        }
        $ql->destruct();
    })
    ->start($start);  
    echo "全部采集完成";     
} catch (Exception $e) {
    file_put_contents('./log.txt', $e->getMessage() . PHP_EOL, FILE_APPEND);
}